#!/bin/sh

. "$SPEC_PATH/abstract/metrics"

e2e_test_extra_hash() {
  "$SHELL" "$PROJECT_PATH/stackgres-k8s/ci/build/build-functions.sh" path_hash \
    "$(realpath --relative-to "$PROJECT_PATH" "$SPEC_PATH/abstract/metrics")"
}

e2e_test_install() {
  OLD_BACKUP_NAME="$(get_sgbackup_name "${SPEC_NAME}-old")"
  BACKUP_NAME="$(get_sgbackup_name "${SPEC_NAME}")"

  install_minio

  create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "2" \
    --set-string cluster.replication.initialization.mode=FromPrimary

  deploy_curl_pod "$CLUSTER_NAMESPACE"

  wait_pods_running "$CLUSTER_NAMESPACE" "4"
  wait_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE"

  cat << EOF | kubectl create -f -
apiVersion: stackgres.io/v1
kind: SGBackup
metadata:
  namespace: $CLUSTER_NAMESPACE
  name: $OLD_BACKUP_NAME
spec:
  sgCluster: $CLUSTER_NAME
EOF
  
  wait_until is_backup_phase "$CLUSTER_NAMESPACE" "$OLD_BACKUP_NAME" "Completed"
}

is_backup_phase() {
  local NAMESPACE="$1"
  local NAME="$2"
  local STATUS="$3"
  [ "$(kubectl get sgbackup -n "$NAMESPACE" "$NAME" -o=jsonpath='{.status.process.status}')" = "$STATUS" ]
}

e2e_test() {
  run_test "Checking that replication is working" check_replication_is_working

  run_test "Checking that replication is working from replica" check_replication_is_working_from_replica

  run_test "Checking that replication is working from backup" check_replication_is_working_from_backup

  if [ "$K8S_DISABLE_VOLUME_SNAPSHOT" != true ] \
    && [ "${E2E_POSTGRES_VERSION%.*}" -ge 15 ] # StackGres snapshot backup only supported for Postgres 15+
  then
    run_test "Checking that replication is working from snapshot backup" check_replication_is_working_from_snapshot_backup
  else
    echo "Skippping Checking that replication is working from snapshot backup"
  fi

  run_test "Checking that replication is working from newly created backup" check_replication_is_working_from_newly_created_backup

  run_test "Checking that metrics are exported" check_metrics
}

check_replication_is_working() {
  check_connectivity -i 0

  local SYNCHRONOUS_STANDBY_NAMES
  SYNCHRONOUS_STANDBY_NAMES="$(kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-0" -c postgres-util -- \
    psql -q -At -c 'SHOW synchronous_standby_names')"
  if echo "$SYNCHRONOUS_STANDBY_NAMES" | grep -q '^$'
  then
    success "async replication is set for primary"
  else
    fail "async replication is not set for primary"
  fi

  local RESULT EXIT_CODE
  try_function run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "CREATE DATABASE test;"
  if "$RESULT"
  then
    success "It's possible to create a database in the primary node"
  else
    fail "It should be possible to create a database in the primary node" 
  fi

  check_service_connectivity -i 0 -h "$CLUSTER_NAME-replicas"
  try_function run_query -p 5432 -i 0 -h "$CLUSTER_NAME-replicas" -q "CREATE TABLE fibonacci(num integer);" -d test > "${LOG_PATH}/test1.log"
  if "$RESULT"
  then
    fail "It's possible to create a table in the replica node"
  else
    success "Good it is not possible to create a table in the replica node" 
  fi

  run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "CREATE TABLE fibonacci(num integer);" -d test
  run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "INSERT INTO fibonacci(num) VALUES (1);" -d test
  run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "INSERT INTO fibonacci(num) VALUES (2);" -d test
  run_query -p 5432 -h "$CLUSTER_NAME" -i 1 -q "INSERT INTO fibonacci(num) VALUES (3);" -d test

  PRIMARY_RESPONSE="$(run_query -p 5432 -i 1 -h "$CLUSTER_NAME" -q "SELECT num FROM fibonacci ORDER BY num;" -d "test")"

  if [ "$(echo "$PRIMARY_RESPONSE" | tr -d '\n')" = "123" ]
  then
    success "inserts on the primary where successful."
  else
    fail "inserts on the primary where not successful."
  fi

  try_function wait_until eval '
    REPLICA_RESPONSE="$(run_query -p 5432 -i 0 -h "$CLUSTER_NAME-replicas" -q "SELECT num FROM fibonacci ORDER BY num;" -d "test")"
    [ "$(echo "$PRIMARY_RESPONSE" | tr -d "\n")" = "$(echo "$REPLICA_RESPONSE" | tr -d "\n")" ]
    '
  if "$RESULT"
  then
    success "replication is working"
  else
    fail "replication is not working. The records don't match between primary and replica for the fibonacci table"
  fi

  if kubectl logs -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1" -c patroni | grep -q "^[^ ]\+ [^ ]\+ INFO: replica has been created using basebackup$"
  then
    success "replication has bootstrapped from basebackup"
  else
    kubectl logs -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1" -c patroni | grep "^[^ ]\+ [^ ]\+ INFO: replica has been created using" || true
    echo
    fail "replication has not bootstrapped from basebackup"
  fi
}

check_replication_is_working_from_replica() {
  create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "3" \
    --set-string cluster.replication.initialization.mode=FromReplica

  wait_pods_running "$CLUSTER_NAMESPACE" "3" "$CLUSTER_NAME-[0-9]"

  PRIMARY_RESPONSE="$(run_query -p 5432 -i 1 -h "$CLUSTER_NAME" -q "SELECT num FROM fibonacci ORDER BY num;" -d "test")"
  try_function wait_until eval '
    REPLICA_RESPONSE="$(run_query -p 5432 -i 0 -h "$CLUSTER_NAME-replicas" -q "SELECT num FROM fibonacci ORDER BY num;" -d "test")"
    [ "$(echo "$PRIMARY_RESPONSE" | tr -d "\n")" = "$(echo "$REPLICA_RESPONSE" | tr -d "\n")" ]
    '
  if "$RESULT"
  then
    success "replication is working"
  else
    fail "replication is not working. The records don't match between primary and replica for the fibonacci table"
  fi

  if kubectl logs -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-2" -c patroni | grep -q "^[^ ]\+ [^ ]\+ INFO: replica has been created using replica_basebackup$"
  then
    success "replication has bootstrapped from replica_basebackup"
  else
    kubectl logs -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-2" -c patroni | grep "^[^ ]\+ [^ ]\+ INFO: replica has been created using" || true
    echo
    fail "replication has not bootstrapped from replica_basebackup"
  fi
}

check_replication_is_working_from_backup() {
  cat << EOF | kubectl create -f -
apiVersion: stackgres.io/v1
kind: SGBackup
metadata:
  namespace: $CLUSTER_NAMESPACE
  name: $BACKUP_NAME
spec:
  sgCluster: $CLUSTER_NAME
EOF
  
  wait_until is_backup_phase "$CLUSTER_NAMESPACE" "$BACKUP_NAME" "Completed"

  create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "2" \
    --set-string cluster.replication.initialization.mode=FromExistingBackup
  wait_pods_terminated "$CLUSTER_NAMESPACE" "2" "$CLUSTER_NAME-[0-9]"
  wait_until kubectl get configmap -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-replication-init" -o json \
    | jq '.data.REPLICATION_INITIALIZATION_BACKUP' -r \
    | grep -q .

  kubectl delete --ignore-not-found -n "$CLUSTER_NAMESPACE" pvc/"$CLUSTER_NAME-data-$CLUSTER_NAME-1" pod/"$CLUSTER_NAME-1" --wait=false
  wait_pods_running "$CLUSTER_NAMESPACE" "2" "$CLUSTER_NAME-[0-9]"

  PRIMARY_RESPONSE="$(run_query -p 5432 -i 1 -h "$CLUSTER_NAME" -q "SELECT num FROM fibonacci ORDER BY num;" -d "test")"
  try_function wait_until eval '
    REPLICA_RESPONSE="$(run_query -p 5432 -i 0 -h "$CLUSTER_NAME-replicas" -q "SELECT num FROM fibonacci ORDER BY num;" -d "test")"
    [ "$(echo "$PRIMARY_RESPONSE" | tr -d "\n")" = "$(echo "$REPLICA_RESPONSE" | tr -d "\n")" ]
    '
  if "$RESULT"
  then
    success "replication is working"
  else
    fail "replication is not working. The records don't match between primary and replica for the fibonacci table"
  fi

  if kubectl logs -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1" -c patroni | grep -q "^[^ ]\+ [^ ]\+ INFO: replica has been created using backup$"
  then
    success "replication has bootstrapped from backup"
  else
    kubectl logs -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1" -c patroni | grep "^[^ ]\+ [^ ]\+ INFO: replica has been created using" || true
    echo
    fail "replication has not bootstrapped from backup"
  fi

  LATEST_BACKUP="$(kubectl get sgbackup -n "$CLUSTER_NAMESPACE" --sort-by='{.metadata.creationTimestamp}' -o name \
    | tail -n 1 | cut -d / -f 2)"
  if [ "x$LATEST_BACKUP" != x ] \
    && kubectl get configmap -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-replication-init" -o json \
    | jq '.data.REPLICATION_INITIALIZATION_BACKUP' -r \
    | grep -qxF "$LATEST_BACKUP"
  then
    success "replication has bootstrapped from latest backup"
  else
    fail "replication has not bootstrapped from latest backup"
  fi
}

check_replication_is_working_from_snapshot_backup() {
  create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" 1 \
    --set cluster.configurations.backups.useVolumeSnapshot=true

  wait_pods_terminated "$CLUSTER_NAMESPACE" "1" "$CLUSTER_NAME-[0-9]"
  
  wait_until eval 'kubectl get cronjob -n "$CLUSTER_NAMESPACE" "$BACKUP_CLUSTER_NAME" -o json \
    | jq ".spec.template.spec.containers[0].env | any(.name == \"USE_VOLUME_SNAPSHOT\" and .value == \"true\")"'

  kubectl delete sgbackup -n "$CLUSTER_NAMESPACE" "$BACKUP_NAME"
  
  cat << EOF | kubectl create -f -
apiVersion: stackgres.io/v1
kind: SGBackup
metadata:
  namespace: $CLUSTER_NAMESPACE
  name: $BACKUP_NAME
spec:
  sgCluster: $CLUSTER_NAME
EOF
  
  wait_until is_backup_phase "$CLUSTER_NAMESPACE" "$BACKUP_NAME" "Completed"

  create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "2" \
    --set-string cluster.replication.initialization.mode=FromExistingBackup
  wait_pods_terminated "$CLUSTER_NAMESPACE" "2" "$CLUSTER_NAME-[0-9]"
  wait_until kubectl get configmap -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-replication-init" -o json \
    | jq '.data.REPLICATION_INITIALIZATION_BACKUP' -r \
    | grep -q .

  kubectl delete --ignore-not-found -n "$CLUSTER_NAMESPACE" pvc/"$CLUSTER_NAME-data-$CLUSTER_NAME-1" pod/"$CLUSTER_NAME-1" --wait=false
  wait_pods_running "$CLUSTER_NAMESPACE" "2" "$CLUSTER_NAME-[0-9]"

  PRIMARY_RESPONSE="$(run_query -p 5432 -i 1 -h "$CLUSTER_NAME" -q "SELECT num FROM fibonacci ORDER BY num;" -d "test")"
  try_function wait_until eval '
    REPLICA_RESPONSE="$(run_query -p 5432 -i 0 -h "$CLUSTER_NAME-replicas" -q "SELECT num FROM fibonacci ORDER BY num;" -d "test")"
    [ "$(echo "$PRIMARY_RESPONSE" | tr -d "\n")" = "$(echo "$REPLICA_RESPONSE" | tr -d "\n")" ]
    '
  if "$RESULT"
  then
    success "replication is working"
  else
    fail "replication is not working. The records don't match between primary and replica for the fibonacci table"
  fi

  if kubectl logs -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1" -c patroni | grep -q "^[^ ]\+ [^ ]\+ INFO: replica has been created using backup$"
  then
    success "replication has bootstrapped from backup"
  else
    kubectl logs -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1" -c patroni | grep "^[^ ]\+ [^ ]\+ INFO: replica has been created using" || true
    echo
    fail "replication has not bootstrapped from backup"
  fi

  LATEST_BACKUP="$(kubectl get sgbackup -n "$CLUSTER_NAMESPACE" --sort-by='{.metadata.creationTimestamp}' -o name \
    | tail -n 1 | cut -d / -f 2)"
  if [ "x$LATEST_BACKUP" != x ] \
    && kubectl get configmap -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-replication-init" -o json \
    | jq '.data.REPLICATION_INITIALIZATION_BACKUP' -r \
    | grep -qxF "$LATEST_BACKUP"
  then
    success "replication has bootstrapped from latest backup"
  else
    fail "replication has not bootstrapped from latest backup"
  fi
}

check_replication_is_working_from_newly_created_backup() {
  BACKUP_END_TIMESTAMP="$(kubectl get sgbackup -n "$CLUSTER_NAMESPACE" "$BACKUP_NAME" -o json \
    | jq '.status.process.timing.end' -r \
    | date -d "$(cat)" +%s)"
  NOW="$(date +%s)"
  if [ "$(( NOW - BACKUP_END_TIMESTAMP ))" -lt 120 ]
  then
    echo "Waiting $(( 120 - NOW + BACKUP_END_TIMESTAMP )) seconds for the existing backups to be enough old"
    sleep "$(( 120 - NOW + BACKUP_END_TIMESTAMP ))"
    NOW="$(date +%s)"
  fi
  create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "2" \
    --set-string cluster.replication.initialization.mode=FromNewlyCreatedBackup \
    --set-string cluster.replication.initialization.backupNewerThan="PT$(( NOW - BACKUP_END_TIMESTAMP ))S"

  kubectl delete --ignore-not-found -n "$CLUSTER_NAMESPACE" pvc/"$CLUSTER_NAME-data-$CLUSTER_NAME-1" pod/"$CLUSTER_NAME-1" --wait=false
  wait_pods_running "$CLUSTER_NAMESPACE" "2" "$CLUSTER_NAME-[0-9]"

  PRIMARY_RESPONSE="$(run_query -p 5432 -i 1 -h "$CLUSTER_NAME" -q "SELECT num FROM fibonacci ORDER BY num;" -d "test")"
  try_function wait_until eval '
    REPLICA_RESPONSE="$(run_query -p 5432 -i 0 -h "$CLUSTER_NAME-replicas" -q "SELECT num FROM fibonacci ORDER BY num;" -d "test")"
    [ "$(echo "$PRIMARY_RESPONSE" | tr -d "\n")" = "$(echo "$REPLICA_RESPONSE" | tr -d "\n")" ]
    '
  if "$RESULT"
  then
    success "replication is working"
  else
    fail "replication is not working. The records don't match between primary and replica for the fibonacci table"
  fi

  if kubectl logs -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1" -c patroni | grep -q "^[^ ]\+ [^ ]\+ INFO: replica has been created using backup$"
  then
    success "replication has bootstrapped from backup"
  else
    kubectl logs -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1" -c patroni | grep "^[^ ]\+ [^ ]\+ INFO: replica has been created using" || true
    echo
    fail "replication has not bootstrapped from backup"
  fi

  NEWLY_CREATED_BACKUP="$(kubectl get sgbackup -n "$CLUSTER_NAMESPACE" -l "app=StackGresCluster,stackgres.io/cluster-name=$CLUSTER_NAME" -o name \
    | cut -d '/' -f 2)"
  if [ "x$NEWLY_CREATED_BACKUP" != x ] \
    && kubectl get configmap -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-replication-init" -o json \
    | jq '.data.REPLICATION_INITIALIZATION_BACKUP' -r \
    | grep -qxF "$NEWLY_CREATED_BACKUP"
  then
    success "replication has bootstrapped from newly created backup"
  else
    fail "replication has not bootstrapped from newly created backup"
  fi
}
